﻿using System.Threading;
using DM4;
using sscore;
using System;
using System.Collections.Generic;
using sscore.Utils;
using System.Text.RegularExpressions;
using System.IO;
using sscore.Requests;
using System.Collections.Specialized;
using System.Net;

namespace AZ
{
    public class AZ : ModuleBase
    {
        private readonly string _searchUrl;
        private readonly string _offenderLink;
        private readonly string _imgHtmlDir;
        private const string StateName = "AZ";
        private ResultFile _imagesResultFile;
        List<string> _zipList;
        const string ZipFileName = "zips_" + StateName + ".txt";
        private StringCollection _idList;

        public AZ()
        {
            //initialize main system constants
            ModuleCode = "SO_AZ";
            Description = "AZ Sex Offenders";
            Version = "1.0";

            //for creating sessions
            StartParam = 0;
            StopParam = 550;

            //initialize help 
            AddToHelp(DM4Consts.StartPosition, String.Format("{0}...{1}", StartParam, StopParam));
            AddToHelp(DM4Consts.EndPositon, String.Format("{0}...{1}", StartParam, StopParam));

            AddToHelp(DM4Consts.IsDownload, "true|false");
            AddToHelp(DM4Consts.IsDownloadImg, "true|false");
            AddToHelp(DM4Consts.InsertToDb, "false|true", "(Expected: servername, dbname)");
            AddToHelp(DM4Consts.ServerName, "");
            AddToHelp(DM4Consts.DbName, "");
            AddToHelp(DM4Consts.IsClearDb, "false|true");
            AddToHelp(DM4Consts.TestParam, "false|true");


            RootUrl = "http://az.gov/";
            _searchUrl = RootUrl + "app/sows/ZipcodeList.xhtml";
            _offenderLink = String.Format("{0}app/sows/SowsOffenders.xhtml?sowsOffendersId={{0}}", RootUrl);
            _imgHtmlDir = RootUrl + "static/sows/images/";
        }

        private void SetParametersDefaults()
        {
            Params.CreateBooleanParam(DM4Consts.IsDownload, true);
            Params.CreateBooleanParam(DM4Consts.IsDownloadImg, true);
            Params.CreateBooleanParam(DM4Consts.InsertToDb, false);
            Params.CreateBooleanParam(DM4Consts.IsClearDb, false);
            Params.CreateStringParam(DM4Consts.DbName, "");
            Params.CreateStringParam(DM4Consts.ServerName, "");
            Params.CreateIntegerParam(DM4Consts.StartPosition, StartParam);
            Params.CreateIntegerParam(DM4Consts.LastPosition, Params.GetIntegerParam(DmConsts.StartPosition));
            Params.CreateIntegerParam(DM4Consts.EndPositon, StopParam);
            Params.CreateIntegerParam(DmConsts.LastRecord, 0); 
            Params.CreateIntegerParam(DmConsts.LastPage, 1); 
        }

        public override OperationResult Init(IDM4Core iCore, DmTaskParams param)
        {
            //use basic initialization sequence
            base.Init(iCore, param);

            SetParametersDefaults();
            return new OperationResult { Result = DM4Consts.Ok };
        }

        public override OperationResult Start()
        {
            DoWork();
            return new OperationResult { Result = DM4Consts.Ok };
        }

        public override OperationResult DoWork()
        {
            OperationResult or = new OperationResult { Result = DM4Consts.ErrorUnknown };

            if (!Ct.IsCancellationRequested)
            {
                WriteLog("Start parameters:", MessageType.Warning);
                foreach (KeyValuePair<string, string> kvp in Params.GetDictionary()) //get all existing parameters
                    WriteLog(String.Format("Key = {0}, Value = {1}", kvp.Key, kvp.Value), MessageType.Warning);

                if (Params.GetBooleanParam(DM4Consts.IsDownload))
                {
                    //create zip list for search
                    _zipList = new List<string>();
                    FileInfo fileInfo = new FileInfo(String.Format("{0}/{1}", SettingsDir, ZipFileName));
                    if (!fileInfo.Exists)
                        if (string.IsNullOrEmpty(Params.GetStringParam(DmConsts.DbName)) || string.IsNullOrEmpty(Params.GetStringParam(DmConsts.ServerName)))
                        {
                            UpdateStatus(DmConsts.Error);
                            or.Message = String.Format("{0} or {1} is empty. Unable to create {2} file", DmConsts.DbName, DmConsts.ServerName, ZipFileName);
                            
                            ResultCode = DM4Consts.SessionResultCodeError;
                            ResultMessage = or.Message;
                            UpdateStatus(DmConsts.Error);
                            WriteErrorLog(or.Message); //writting to error and session log   
                            return or ;
                        }
                        else
                            Offenders.GetZipFile(StateName, Params.GetStringParam(DmConsts.DbName), Params.GetStringParam(DmConsts.ServerName), SettingsDir);

                    fileInfo = new FileInfo(String.Format("{0}/{1}", SettingsDir, ZipFileName));
                    if (fileInfo.Exists)
                    {
                        try
                        {
                            using (StreamReader fileReader = fileInfo.OpenText())
                            {
                                String line;
                                while ((line = fileReader.ReadLine()) != null)
                                    _zipList.Add(line.Trim());
                                fileReader.Close();
                            }
                        }
                        catch (Exception ex)
                        {
                            UpdateStatus(DmConsts.Error);
                            or.Message = String.Format("Unable to read {0} file: {1}", ZipFileName, ex.Message);
                            WriteErrorLog(or.Message);
                            return or;
                        }
                    }
                    else
                    {
                        UpdateStatus(DmConsts.Error);
                        or.Message = String.Format("File {0} not found", fileInfo.FullName);
                        WriteErrorLog(or.Message); //writting to error and session log 
                        return or;
                    }
                    Load();
                }

                if (Params.GetBooleanParam(DM4Consts.InsertToDb))
                    if ((Params.GetStringParam(DM4Consts.DbName) == "") || (Params.GetStringParam(DM4Consts.ServerName) == ""))
                    {
                         or.Message = String.Format("{0} or {1} is empty", DM4Consts.DbName, DM4Consts.ServerName);
                        ResultCode = DM4Consts.SessionResultCodeError;
                        ResultMessage = or.Message;
                        UpdateStatus(DmConsts.Error);
                        WriteErrorLog(or.Message); //writting to error and session log   

                        return or;
                    }
                    else
                    {
                        //insert offenders to db 
                        UpdateStatus(DM4Consts.SessionInsertToDb);

                        Offenders.OffInsertToDatabase(StateName, Params.GetStringParam(DmConsts.DbName),
                             Params.GetStringParam(DmConsts.ServerName), ResultDir, Params.GetBooleanParam(DmConsts.IsClearDb),
                             WriteLog);

                    }

                //check if need abort task
                if (Ct.IsCancellationRequested)
                {
                    ResultCode = DM4Consts.SessionResultTerminateByUser;
                    ResultMessage = "Session terminated by user";
                    UpdateStatus(DM4Consts.SessionWait);
                    WriteLog("Session aborted by user", MessageType.Cool);
                }
                else
                {
                    UpdateProgress(1);
                    ResultCode = DM4Consts.SessionResultCodeOk;
                    ResultMessage = String.Format("Session complete, errors : {0}, records : {1}", TaskParams.ErrorCount, TaskParams.RecordCount);
                    UpdateStatus(DM4Consts.SessionComplete);
                    WriteLog("------------Finish----------------", MessageType.Cool);
                }
            }
            else
            {
                ResultCode = DM4Consts.SessionResultTerminateByUser;
                ResultMessage = "Session terminated by user";
                UpdateStatus(DM4Consts.SessionWait);
                WriteLog("Session aborted by user", MessageType.Cool);
            }


            return or;
        }

        private void Load()
        {
            string resultFileName = String.Format("{0}\\{1}.{2}", ResultDir, TaskParams.SessionId, DmConsts.ResultFilesExtension);
            bool workFlag = true;

            ResultFile = new ResultFile(resultFileName, OffendersResult.GetHeader());
            _imagesResultFile = new ResultFile(String.Format("{0}\\{1}_images.{2}", ResultDir, TaskParams.SessionId, DM4Consts.ResultFilesExtension), "FileName|FileUrl");
            try
            {
                while (workFlag)
                {
                    //рабочий цикл
                    string error;

                    try
                    {
                        error = "";  //Required!!  /*if (!String.IsNullOrEmpty(error)) --> restart thread; else --> download finished*/

                        WriteLog("Start download...");
                        Searching();
                    }
                    catch (Exception e)
                    {
                        error = e.Message;
                    }

                    if (!String.IsNullOrEmpty(error))
                    {
                        Thread.Sleep(30000);
                        WriteErrorLog(error);
                        WriteLog("Try continue work...", MessageType.Warning);
                    }
                    else
                    {
                        workFlag = false;  //все нормально откачало, заканчиваем работу
                    }
                }
            }
            finally
            {
                ResultFile.Close();
                _imagesResultFile.Close();
            }
        }

        private string GetCookies()
        {
            WriteLog("get cookies");
            string html = GetRequest.ReturnCookies(_searchUrl, Rpm);
            Rpm.Cookies = RegExpr.Parse(Rpm.Cookies, @"(?si:^([^;]*?);)");
            return html;
        }

        private void Searching()
        {
            if (Test)
            {
                Params.SetIntegerParam(DmConsts.LastPosition, Params.GetIntegerParam(DmConsts.StartPosition));
                Params.SetIntegerParam(DmConsts.EndPositon, Params.GetIntegerParam(DmConsts.StartPosition));
            }

            if (Params.GetIntegerParam(DmConsts.EndPositon) > (_zipList.Count - 1))
                Params.SetIntegerParam(DmConsts.EndPositon, _zipList.Count - 1);

            GetCookies();
            for (int indSearchParam = Params.GetIntegerParam(DmConsts.LastPosition); indSearchParam <= Params.GetIntegerParam(DmConsts.EndPositon); indSearchParam++)
            {
                //check if need abort task
                if (Ct.IsCancellationRequested)
                    break;

                Params.SetIntegerParam(DmConsts.LastPosition, indSearchParam);
                UpdateProgress();

                WriteLog(String.Format("Process search param [{0}][{1}].",
                    Params.GetIntegerParam(DmConsts.LastPosition),
                    _zipList[Params.GetIntegerParam(DmConsts.LastPosition)]));

                string html = GetRequest.WithCookies(String.Format(_searchUrl, Params.GetIntegerParam(DmConsts.LastPosition)), Rpm);
                string javaxFacesViewState = HtmlUtils.GetServerVariableValueById("javax.faces.ViewState", html);
                html = PostRedirectRequest.RefererWithCookies(String.Format("{0};{1}", _searchUrl, Rpm.Cookies), _searchUrl,
                    String.Format("zipcodeSearch=zipcodeSearch&zipcodeSearch%3AzipDecoration%3Aid={0}&h=ieExplores&zipcodeSearch%3Asearch=Search&javax.faces.ViewState={1}", 
                        _zipList[Params.GetIntegerParam(DmConsts.LastPosition)], javaxFacesViewState),
                    Rpm);


                string checkString = RegExpr.Parse(html, String.Format(@"(?si:<h2>(([\d]+) Sex Offenders located in {0})</h2>)", _zipList[Params.GetIntegerParam(DmConsts.LastPosition)]));
                if (!string.IsNullOrEmpty(checkString))
                {
                    WriteLog(checkString);      
                    CollectIds(html);
                }
                else if (RegExpr.IsMatch(html, String.Format(@"(?si:No Sex Offenders Found Within {0})", _zipList[Params.GetIntegerParam(DmConsts.LastPosition)])))
                    WriteLog("No records found");
                else 
                    throw new Exception(DmConsts.ExcNeedRestart + " [No search result in Searching() function]");
            }
        }

        private void CollectIds(string html)
        {
            _idList = new StringCollection();
            MatchCollection mc = RegExpr.ParseMatchCollection(html, @"(?si:<div[^<>]*?>\s*<h2>\s*([^<>]+)\s*</h2>\s*<table(.*?)id=""(myForm[^\x3A""]*?\x3Atable[^\x3A]*?\x3As[^_""]+)""(.*?)</table>)");

            foreach (Match m in mc)
            {
                string tableCaption = m.Groups[1].Value.Trim();
                //string tableContent = m.Groups[2].Value.Trim();
                string tableId = m.Groups[3].Value.Trim();
                //string navigationDiv = m.Groups[4].Value.Trim();

                //navigate by pages in table
                if (!RegExpr.IsMatch(tableCaption, @"(?si:located outside)"))
                    NavigateByPagesInResultTable(html, tableId, tableCaption);
            }

            NavigateByOffendersId();
        }

        private void NavigateByPagesInResultTable(string html, string tableId, string tableCaption)
        {
            WriteLog(String.Format("process table [{0}][{1}]", tableCaption, tableId));
            int iPage = 1;
            do
            {
                if (Ct.IsCancellationRequested)
                    break;

                CollectId(html, tableId, tableCaption);
                if (RegExpr.IsMatch(html, String.Format(@"(?si:'{0}':'fastforward')", tableId)))
                {
                    iPage++;
                    WriteLog(String.Format("Navigate to page [{0}]", iPage));

                    string cid = RegExpr.Parse(html, @"(?si:[\x3F;]+cid=([^""]+)"")");
                    string javaxFacesViewState = HtmlUtils.GetServerVariableValueById("javax.faces.ViewState", html);

                    html = PostRedirectRequest.RefererWithCookies(RootUrl + "app/sows/Zipcode.xhtml?javax.portlet.faces.DirectLink=true",
                         String.Format("{0}app/sows/Zipcode.xhtml?zipcodeId={1}&orderUM=lastName+asc&orderIn=lastName+asc&orderUV=lastName+asc&orderOut=lastName+asc&cid={2}",
                            RootUrl, _zipList[Params.GetIntegerParam(DmConsts.LastPosition)], cid),
                         String.Format(String.Format("AJAXREQUEST=_viewRoot&{0}={0}&javax.faces.ViewState={{0}}&{{1}}=fastforward&", 
                            RegExpr.Parse(tableId, @"(?si:^([^:]+):)")),
                                     javaxFacesViewState, tableId.Replace(":", "%3A")),
                         Rpm);
                }
                else break;
            } while (true);
        }

        private void CollectId(string html, string tableId, string tableCaption)
        {
            MatchCollection mc = RegExpr.ParseMatchCollection(
                RegExpr.Parse(html,
                    String.Format(@"(?si:<div[^<>]*?>\s*<h2>\s*{0}\s*</h2>\s*<table.*?id=""{1}"".*?</table>(.*?)</table>)", tableCaption, tableId))
                , @"(?si:SowsOffenders.xhtml\x3FsowsOffendersId=([^\x26]+)\x26)");

            if (mc.Count == 0 && !RegExpr.IsMatch(html,"(?si:Click on map markers for more information)"))
                mc = RegExpr.ParseMatchCollection(html, @"(?si:SowsOffenders.xhtml\x3FsowsOffendersId=([^\x26]+)\x26amp;)");

            WriteLog(String.Format("found [{0}] IDs matches", mc.Count));

            int i = 0;
            foreach (Match matchLoc in mc)
                if (!_idList.Contains(matchLoc.Groups[1].Value))
                {
                    _idList.Add(matchLoc.Groups[1].Value);
                    i++;
                }

            WriteLog(String.Format("[{0}] unique offenders IDs added to _idList", i));
        }

        private void NavigateByOffendersId()
        {
            WriteLog(String.Format("total id(s) found: [{0}]", _idList.Count));
            int iOffender = 0;
            foreach (string lookupKey in _idList)
            {
                //check if need abort task
                if (Ct.IsCancellationRequested)
                    break;

                if (iOffender >= Params.GetIntegerParam(DmConsts.LastRecord))
                {
                    string offenderHtml = GetRequest.WithCookies(String.Format(_offenderLink, lookupKey), Rpm);
                    WriteLog(String.Format("ParseOffender [{0}][{1}] ", Params.GetIntegerParam(DmConsts.LastRecord), lookupKey));

                    if (!RegExpr.IsMatch(offenderHtml,@"(?si:offenderContent\:name)")) 
                        throw new Exception(DmConsts.ExcNeedRestart + " [Offender page]");                    
                    ParseOffender(offenderHtml, lookupKey);
                    UpdateRecordCount(1);
                    Params.SetIntegerParam(DmConsts.LastRecord, Params.GetIntegerParam(DmConsts.LastRecord) + 1);
                }
                iOffender++;
            }
            Params.SetIntegerParam(DmConsts.LastRecord, 0);
        }

        private void ParseOffender(string offenderHtml, string sourceId)
        {
            offenderHtml = offenderHtml.Replace("\r\n", "").Replace("\t", "").Replace("\n", "").Replace("\r", "").Replace("&nbsp;", " ");

            using (OffendersResult or = new OffendersResult())
            {
                or.Source_State = StateName;
                or.Source_id = sourceId;
                or.offenderLink = String.Format(_offenderLink, sourceId);

                or.fullname = RegExpr.Parse(offenderHtml, @"(?si:offenderContent:name"">(.*?)</span>)").Trim();
                or.Address = RegExpr.Parse(offenderHtml, @"(?si:offenderContent:address"">(.*?)</span>)").Trim();
                or.Address += "@" + RegExpr.Parse(offenderHtml, @"(?si:offenderContent:city"">(.*?)</span>)").Trim();
                or.Age = RegExpr.Parse(offenderHtml, @"(?si:offenderContent:ageVal"">(.*?)</span>)").Trim();
                or.Ethnicity = RegExpr.Parse(offenderHtml, @"(?si:offenderContent:raceVal"">(.*?)</span>)").Trim();
                or.eye_color = RegExpr.Parse(offenderHtml, @"(?si:offenderContent:eyesVal"">(.*?)</span>)").Trim();
                or.hair_color = RegExpr.Parse(offenderHtml, @"(?si:offenderContent:hairVal"">(.*?)</span>)").Trim();
                or.height = RegExpr.Parse(offenderHtml, @"(?si:offenderContent:heightVal"">(.*?)</span>)").Trim();
                or.weight = RegExpr.Parse(offenderHtml, @"(?si:offenderContent:weightVal"">(.*?)</span>)").Trim();
                or.sex = RegExpr.Parse(offenderHtml, @"(?si:offenderContent:sexVal"">(.*?)</span>)").Trim();
                or.Risk = RegExpr.Parse(offenderHtml, @"(?si:offenderContent:level"">(.*?)</span>)").Trim();

                MatchCollection mc = RegExpr.ParseMatchCollection(offenderHtml, @"(?si:offenderContent:offenseVal"">(.*?)</span>)");
                if (mc.Count > 0) or.Crime += "CrimeCaption;Convicting Jurisdiction~";
                for (int i = 0; i < mc.Count; i++)
                {
                    try
                    {
                        or.Crime += RegExpr.ParseMatchCollection(offenderHtml, @"(?si:offenderContent:offenseVal"">(.*?)</span>)")[i].Groups[1].Value.Trim() + ";";
                    }
                    catch { or.Crime += ";"; }
                    try
                    {
                        or.Crime += RegExpr.ParseMatchCollection(offenderHtml, @"(?si:offenderContent:jurisdictionVal"">(.*?)</span>)")[i].Groups[1].Value.Trim() + "~";
                    }
                    catch { or.Crime += "~"; }
                }
                string htmlImageId = RegExpr.Parse(offenderHtml, String.Format(@"(?si:{0}([\S]*?\x2Ejpg))", _imgHtmlDir));

                if (Params.GetBooleanParam(DmConsts.IsDownloadImg) && !string.IsNullOrEmpty(htmlImageId))
                {
                    or.Image_ID = Guid.NewGuid() + ".jpg";
                    try
                    {
                        or.ImageLink = _imgHtmlDir + htmlImageId;
                        Offenders.DownloadImageWithExceptions(_imagesResultFile, String.Format("{0}\\{1}\\{2}", ResultDir, DmConsts.PhotosDir, or.Image_ID), or.ImageLink, TaskParams.SessionId);
                    }
                    catch (WebException webEx)
                    {
                        if (((HttpWebResponse)webEx.Response).StatusCode == HttpStatusCode.NotFound
                            || ((HttpWebResponse)webEx.Response).StatusCode == HttpStatusCode.InternalServerError)
                        {
                            or.Image_ID = "";
                            or.ImageLink = "";
                        }
                        else throw new Exception("error in downloading offenders image", webEx);
                    }
                }

                ResultFile.Write(or.GetValues());
            }
        }

        private void UpdateProgress()
        {
            if (Params.GetIntegerParam(DmConsts.StartPosition) > Params.GetIntegerParam(DmConsts.EndPositon))
                throw new Exception(String.Format("Invalid parameters {2}[{0}]; {3} [{1}]",
                    Params.GetIntegerParam(DmConsts.StartPosition), Params.GetIntegerParam(DmConsts.EndPositon),
                    DmConsts.StartPosition, DmConsts.EndPositon));
            if (Params.GetIntegerParam(DmConsts.LastPosition) > Params.GetIntegerParam(DmConsts.EndPositon))
                throw new Exception(String.Format("Invalid parameters {2}[{0}]; {3} [{1}]",
                    Params.GetIntegerParam(DmConsts.LastPosition), Params.GetIntegerParam(DmConsts.EndPositon),
                    DmConsts.StartPosition, DmConsts.EndPositon));
            if (Params.GetIntegerParam(DmConsts.StartPosition) > Params.GetIntegerParam(DmConsts.LastPosition))
                throw new Exception(String.Format("Invalid parameters {2}[{0}]; {3} [{1}]",
                    Params.GetIntegerParam(DmConsts.StartPosition), Params.GetIntegerParam(DmConsts.LastPosition),
                    DmConsts.StartPosition, DmConsts.LastPosition));
            UpdateProgress(((double)(Params.GetIntegerParam(DmConsts.LastPosition) - Params.GetIntegerParam(DmConsts.StartPosition))) / (Params.GetIntegerParam(DmConsts.EndPositon) - Params.GetIntegerParam(DmConsts.StartPosition) + 1));
        }
    }
}
